<?php
namespace frontend\controllers;

use common\models\ImageTypeModel;
use common\models\LianTypeModel;
use common\models\LianWebModel;
use common\models\NovelModel;
use common\models\NovelTypeModel;
use common\models\SetAuthorModel;
use frontend\models\SmsCodeModel;
use Yii;
use yii\data\Pagination; //分页
use frontend\base\BaseController;
use frontend\utils\GetUrl;
use frontend\utils\UrlUtil;
use frontend\utils\CurlUtil;

/**
 * Collection controller
 */
class CollectionController extends BaseController
{
    public $name = '';
    public $type = '';
    public static $novel_type='';

    public function beforeAction($action)
    {
        $this->name = $this->post('name','');
        $this->type = $this->post('type','');
        return true;
    }

    /**
     * 欢迎页
     * @return string
     */
    public function actionDemo()
    {
        $str = file_get_contents('https://www.7dadi.com/webdir/zuopai/109-5.html');
        $par = '/sitelist(.*)<\/ul>/Uis';
        preg_match($par, $str, $arr);
        var_dump($arr);die;
        if(!empty($arr[1])){
            $par = '/.*src="(.*)".*alt="(.*)".*<p>(.*)<\/p>.*">(.*)<\/a>/Uis';
            preg_match_all($par,$arr[1],$arr2);
            var_dump($arr2);exit;
        }
        echo '测试成功,欢迎使用!!!'.'<br><br><br>';
        echo "['collection-star'=>'传参数name，采集name对应的网站信息,type=1一级(类型)  2下级']
                <a href='/collection/collection-star?name=shubao&type=1' target='_blank' style='text-decoration:none;'>   去采集吧 >></a>".'<br>';
    }

    /**
     * 爬虫
     * @return string
     */
    public function actionCollectionNovel()
    {
        ini_set("max_execution_time", "36000");
        Yii::$app->layout = false;
        $name = $this->name;
        $type = $this->type;
        $url = Yii::$app->params['collection_site'][$name]['url'];
        $match = Yii::$app->params['collection_site'][$name]['type'];
        $replace = Yii::$app->params['collection_site'][$name]['replace'];
        if($type && $type == 2){
            $match = Yii::$app->params['collection_site'][$name]['content'];
            $key_url = NovelTypeModel::find()->select(['url','id'])->where(['key'=>$this->name])->asArray()->all();
            $result = $res = [];
            if(!empty($key_url)){
                foreach ($key_url as $k=>$val){
                    if($k != 2){
                        continue;
                    }
                    $page = NovelModel::find()->select(['page'])->where(['type'=>$val['id']])->orderBy('page desc')->asArray()->one();
                    $page = !empty($page) ? $page['page']+1 : 1;
                    for ($i=$page;;$i++){
                        if($this->name != 'xiashu'){
                            $url = str_replace('type',$k+1,str_replace('num',$i,Yii::$app->params['collection_site'][$name]['next_data']));
                        }else{
                            $url = str_replace('num',$i,Yii::$app->params['collection_site'][$name]['next_data']);
                        }
                        $url = $val['url'].$url;
                        $result = UrlUtil::start($url, $match, $replace);usleep(100);
                        if(!empty($result)){
                            foreach ($result as $key=>$value){
                                $result[$key]['novel_type'] = $val['id'];
                                $result[$key]['page'] = $i;
                            }
                        }
                        $this->insertContent($this->resetContent($result));
                    }
                }
            }
            if(!empty($res)){
                foreach ($res as $k=>$val){
                    if(!empty($val)){
                        foreach ($val as $value){
                            $value['novel_type'] = $k;
                            $result[] = $value;
                        }
                    }
                }
            }
        }else{
            $result = UrlUtil::start($url, $match, $replace);
            $this->insertContent($this->resetContent($result));
        }
    }

    //数据整理
    public function resetContent($result)
    {
        if(!empty($result)){
            foreach ($result as $k => $val){
                if($this->name == 'jianpan'){
                    $result[$k]['url'] = $val['title'];
                    preg_match('/[\x{4e00}-\x{9fff}]+/u', $val['url'], $matches);
                    $title = mb_substr(strrchr($val['url'],'/'),1,mb_strlen($val['url']));
//                    $result[$k]['title'] = !empty($matches) ? $matches[0] : '';
                    $result[$k]['title'] = $title;
                }
                if($this->name == 'shubao'){
                    if($val['title'] == '电子书' || empty($val['title'])){
                        unset($result[$k]);continue;
                    }
                    $url = explode('//',$val['url']);
                    $result[$k]['url'] = 'https://www.bookbao8.com/'.$url[2];
                }
                if($this->name == 'qingdou'){
                    $result[$k]['url'] = 'http://www.qingdoutxt.com'.$val['url'];
                }
            }
        }
//        echo '<pre>';
//        print_r($result);die;
        return $result;
    }

    /**
     * 爬虫数据入库
     */
    public function insertContent($content)
    {
        $type = NovelTypeModel::find()->select(['source'])->where(['key'=>$this->name])->asArray()->one();
        if(!empty($type)){
            $source = $type['source'];
        }else{
            $type = NovelTypeModel::find()->select(['source'])->orderBy('source desc')->asArray()->one();
            if(!empty($type)) {
                $source = $type['source'] + 1;
            }else{
                $source = 1;
            }
        }
        if($this->type == 2) { //详细信息入库
            $type_data = [];
            if ($content) {
                foreach ($content as $k => $val) {
                    $title = str_replace('"','',$val['title']);
                    if(mb_strlen($title) >= 20) continue;
                    $url = str_replace('"','',$val['url']);
                    $count = NovelModel::find()->where(['title' => $title,'source'=>$source,'type'=>$val['novel_type']])->count();
                    if($count == 0) {
                        $type_data[] = ['title' => $title, 'source' => $source, 'url' => $url, 'type' => $val['novel_type'], 'page' => $val['page']];
                    }
                }
            }
            $model = NovelModel::tableName();
            $attr = ['title', 'source', 'url','type','page'];
            $data = $type_data;
        }else{  //类型信息入库
            $type_data = [];
            if ($content) {
                foreach ($content as $k => $val) {
                    $title = str_replace('"','',$val['title']);
                    if(mb_strlen($title) >= 20) continue;
                    $url = str_replace('"','',$val['url']);
                    $count = NovelTypeModel::find()->where(['title' => $title,'source'=>$source,'key'=>$this->name,'url'=>$url])->count();
                    if($count == 0) {
                        $type_data[] = ['title' => $title, 'source' => $source, 'url' => $url, 'key' => $this->name];
                    }
                }
            }
            $model = NovelTypeModel::tableName();
            $attr = ['title', 'source', 'url','key'];
            $data = $type_data;
        }
        if(!empty($data)) {
            $res = Yii::$app->db->createCommand()->batchInsert($model, $attr, $data)->execute();
        }
    }

    /*
     * 图片爬虫
     */
    public function actionCollectionImage()
    {
        if($this->name == 'shetu'){
            $this->insertSheTu();
        }
    }

    //摄图网
    public function insertSheTu()
    {
        if($this->type == 1) {
            $type_data = [];
            $image_preg = Yii::$app->params['collection_image'][$this->name]['type'];
            $image_str = file_get_contents('http://699pic.com/');
            preg_match_all($image_preg,$image_str,$image_source);
            unset($image_source[0],$image_source[1]);
            $image_source = UrlUtil::resetContent($image_source,2,3);
            if ($image_source) {
                foreach ($image_source as $k => $val) {
                    $title = str_replace('"','',$val['title']);
                    if(mb_strlen($title) >= 20) continue;
                    $url = str_replace('"','',$val['url']);
                    if(strpos($url,'-1') ==false){
                        $url = str_replace('.html','-1/',$val['url']);
                    }
                    $count = NovelTypeModel::find()->where(['title' => $title,'source'=>1,'key'=>$this->name,'url'=>$url])->count();
                    if($count == 0) {
                        $type_data[] = ['title' => $title,'source'=>1,'url'=>$url,'key'=>$this->name];
                    }
                }
            }
            $model = ImageTypeModel::tableName();
            $attr = ['title', 'source', 'url','key'];
            $data = $type_data;
        }else{
            $image_preg = Yii::$app->params['collection_image'][$this->name]['content'];
        }
        $res = Yii::$app->db->createCommand()->batchInsert($model, $attr, $data)->execute();
    }

    /*
     * 小说作家
     */
    public function actionSetAuthor()
    {
        ini_set("max_execution_time", "3600");
        if($this->name == 'jianpan'){
            $source = 1;
        }
        if($this->name == 'shubao'){
            $source = 2;
        }
        $count = NovelModel::find()->select(['url','id'])->where(['source'=>$source,'author'=>''])->count();
        if($count == 0){
            echo '当前资源已经用尽!!!';die;
        }
        for ($i=1;$i<=floor($count/100);$i++) {
            $url_all = NovelModel::find()->select(['url', 'id','title'])->where(['source' => $source,'author'=>''])
                ->offset(($i-1)*100)->limit(100)->orderBy('page asc')->asArray()->all();
            if (!empty($url_all)) {
                foreach ($url_all as $val) {
                    $str = file_get_contents($val['url']);
                    $par = Yii::$app->params['collection_site'][$this->name]['author'];
                    preg_match_all($par, $str, $arr);
                    if(empty($arr[1][0])){
                        continue;
                    }
                    $author = str_replace('"', '', $arr[1][0]);
//                    $data[] = ['source'=>$source,'title'=>$val['title'],'author'=>$author];
                    NovelModel::updateAll(['author' => $author], ['id' => $val['id']]);
                    usleep(20);
                }
            }
//            $res = Yii::$app->db->createCommand()->batchInsert(SetAuthorModel::tableName(), ['source','title','author'], $data)->execute();
//            $sql = "UPDATE novel n INNER JOIN set_author a ON n.`title`=a.`title` AND n.`source`=a.`source` SET n.author=a.author";
//            Yii::$app->db->createCommand($sql)->execute();
        }

    }


    /*
     * 青豆网
     */
    public function actionQingdou()
    {
        ini_set("max_execution_time", "36000");
        $key_url = NovelTypeModel::find()->select(['url','id'])->where(['key'=>'qingdou'])->asArray()->all();
        $result = $res = [];
        $type_page = $this->post('type',0);
        $next_data = Yii::$app->params['collection_site']['qingdou']['next_data'];
        $match = Yii::$app->params['collection_site']['qingdou']['content'];
        if(!empty($key_url)){
            foreach ($key_url as $k=>$val){
                if($k <= $type_page){
                    continue;
                }
                $page = NovelModel::find()->select(['page'])->where(['type'=>$val['id']])->orderBy('page desc')->asArray()->one();
                $page = !empty($page) ? $page['page']+1 : 1;
                for ($i=$page;;$i++){
                    $url = str_replace('num',$i,$next_data);
                    $url = $val['url'].$url;
                    $str = file_get_contents($url);
                    preg_match_all($match,$str,$content);
                    $sub_count = count($content[1]);
                    for ($j=0;$j < $sub_count;$j++){
                        $result[$j]['url'] = 'http://www.qingdoutxt.com'.str_replace('"', '', $content[1][$j]);
                        $result[$j]['title'] = str_replace('"', '', $content[2][$j]);
                        $result[$j]['author'] = str_replace(' ', '', str_replace('"', '', $content[3][$j]));
                    }
                    $type_data = [];
                    if ($result) {
                        foreach ($result as $kk => $value) {
                            $title = $value['title'];
                            if(mb_strlen($title) >= 20) continue;
                            $url = $value['url'];
                            $count = NovelModel::find()->where(['title' => $title,'source'=>3,'type'=>$val['id']])->count();
                            if($count == 0) {
                                $type_data[] = ['title' => $title, 'author'=>$value['author'],'source' => 3, 'url' => $url, 'type' => $val['id'], 'page' => $i];
                            }
                        }
                    }
                    $model = NovelModel::tableName();
                    $attr = ['title', 'author', 'source', 'url','type','page'];
                    $res = Yii::$app->db->createCommand()->batchInsert($model, $attr, $type_data)->execute();
                    usleep(100);
                }
            }
        }
    }

    /*
     * 下书网
     */
    public function actionXiashu()
    {
        ini_set("max_execution_time", "36000");
        if($this->type == 1) {
            $str = file_get_contents('https://www.xiashu.la');
            $par = '/<li><a.*href="(.*)">(.*)<\/a><span.*class="line"><\/span><\/li>/';
            preg_match_all($par, $str, $type);
            $sub_count = count($type[1]);
            $result = [];
            for ($j=0;$j < $sub_count;$j++){
                $result[$j]['url'] = 'https://www.xiashu.la'.str_replace('"', '', $type[1][$j]);
                $result[$j]['title'] = str_replace('"', '', $type[2][$j]);
            }
            $type_data = [];
            if ($result) {
                foreach ($result as $kk => $value) {
                    $title = $value['title'];
                    if(mb_strlen($title) >= 20) continue;
                    $url = $value['url'];
                    $count = NovelTypeModel::find()->where(['title' => $title,'source'=>4,'key'=>'xiashu','url'=>$url])->count();
                    if($count == 0) {
                        $type_data[] = ['title' => $title, 'source' => 4, 'url' => $url, 'key' => 'xiashu'];
                    }
                }
            }
            $model = NovelTypeModel::tableName();
            $attr = ['title', 'source', 'url','key'];
            $res = Yii::$app->db->createCommand()->batchInsert($model, $attr, $type_data)->execute();
            echo '类型添加完毕';
        }else{
            $key_url = NovelTypeModel::find()->select(['url','id'])->where(['key'=>'xiashu'])->asArray()->all();
            $result = $res = [];
            $next_data = Yii::$app->params['collection_site']['xiashu']['next_data'];
            $match = Yii::$app->params['collection_site']['xiashu']['content'];
            if(!empty($key_url)){
                foreach ($key_url as $k=>$val){
                    $page = NovelModel::find()->select(['page'])->where(['type'=>$val['id']])->orderBy('page desc')->asArray()->one();
                    $page = !empty($page) ? $page['page']+1 : 1;
                    for ($i=$page;;$i++){
                        $url = str_replace('num',$i,$next_data);
                        $url = $val['url'].$url;
                        $str = file_get_contents($url);
                        preg_match_all($match,$str,$content);
                        $sub_count = count($content[1]);
                        for ($j=0;$j < $sub_count;$j++){
                            $result[$j]['url'] = 'https://www.xiashu.la'.str_replace('"', '', $content[1][$j]);
                            $result[$j]['title'] = str_replace('"', '', $content[2][$j]);
                            $result[$j]['author'] = str_replace(' / 著', '', str_replace('"', '', $content[3][$j]));
                        }
                        $type_data = [];
                        if ($result) {
                            foreach ($result as $kk => $value) {
                                $title = $value['title'];
                                if(mb_strlen($title) >= 20) continue;
                                $url = $value['url'];
                                $count = NovelModel::find()->where(['title' => $title,'source'=>4,'type'=>$val['id']])->count();
                                if($count == 0) {
                                    $type_data[] = ['title' => $title, 'author'=>$value['author'],'source' => 4, 'url' => $url, 'type' => $val['id'], 'page' => $i];
                                }
                            }
                        }
                        $model = NovelModel::tableName();
                        $attr = ['title', 'author', 'source', 'url','type','page'];
                        $res = Yii::$app->db->createCommand()->batchInsert($model, $attr, $type_data)->execute();
                        usleep(100);
                    }
                }
            }
        }
    }


    /*
     * 趣链
     */
    public function actionLiantype()
    {
        ini_set("max_execution_time", "36000");
        $level = $this->type;
        $lian_type = LianTypeModel::find()->select(['url','id'])->where(['type'=>$level])->asArray()->all();
        if(!empty($lian_type)){
            foreach ($lian_type as $k=>$val){
//                if($k<4){
//                    continue;
//                }
                $str = file_get_contents($val['url']);
                $par = '/scatelist(.*)<\/div>/Uis';
                preg_match($par,$str,$arr);
                if(!empty($arr[1])){
                    $par = '/href="(.*)".*>(.*)<\/a/Uis';
                    preg_match_all($par,$arr[1],$arr2);
                    $sub_count = count($arr2[1]);
                    $result = [];
                    for ($j=0;$j < $sub_count;$j++){
                        $result[$j]['url'] = str_replace('"', '', $arr2[1][$j]);
                        $result[$j]['title'] = str_replace('"', '', $arr2[2][$j]);
                    }
                    if ($result) {
                        foreach ($result as $kk => $value) {
                            $title = $value['title'];
                            $url = $value['url'];
                            $count = LianTypeModel::find()->where(['type'=>2])->andWhere(['like','name',$title])->one();
                            if(empty($count)) {
                                $type_data[] = ['name' => $title,'url'=>$url];
                            }else{
                                LianTypeModel::updateAll(['url'=>$url],['type'=>2,'id'=>$count->id]);
                            }
                        }
//                        $model = LianTypeModel::tableName();
//                        $attr = ['name', 'parent_id', 'type','url'];
//                        $res = Yii::$app->db->createCommand()->batchInsert($model, $attr, $type_data)->execute();
                    }
                }
            }
            echo '类型添加完毕';
        }
    }

    public function actionLian()
    {
        ini_set("max_execution_time", "36000");
        $lian_type = parent::redisGet('lian_type',true);
        if(empty($lian_type)) {
            $lian_type = LianTypeModel::find()->select(['id','url'])->where(['type' => 2])->andWhere(['like','url','7dadi'])->asArray()->all();
            parent::redisSet('lian_type', $lian_type);
        }
        if(!empty($lian_type)){
            foreach ($lian_type as $k=>$val){
//                if($k <= 265){
//                    continue;
//                }
                for ($i=1;;$i++) {
                    $url = str_replace('-1.html','-'.$i.'.html',$val['url']);
                    $str = file_get_contents($url);usleep(2000);
                    $par = '/sitelist(.*)<\/ul>/Uis';
                    preg_match($par, $str, $arr);
                    if (!empty($arr[1])) {
                        $par = '/.*src="(.*)".*alt="(.*)".*<p>(.*)<\/p>.*">(.*)<\/a>/Uis';
                        preg_match_all($par, $arr[1], $arr2);
                        if(empty($arr2[1])){
                            echo $k.'--当前类型已加载完毕->'.$url.'<br>';break;
                        }
                        $sub_count = count($arr2[1]);
                        $type_data = $result = [];
                        for ($j=0;$j < $sub_count;$j++){
                            $result[$j]['image'] = $arr2[1][$j];
                            $result[$j]['desc'] = $arr2[3][$j];
                            $result[$j]['url'] = $arr2[4][$j];
                            $result[$j]['title'] = $arr2[2][$j];
                        }
                        if ($result) {
                            foreach ($result as $kk => $value) {
                                $count = LianWebModel::find()->where(['type'=>$val['id'],'title' => $value['title'],'url'=>$value['url']])->count();
                                if($count == 0) {
                                    $type_data[] = ['type'=>$val['id'],'title' => $value['title'],'url'=>$value['url'],'image'=>$value['image'],'desc'=>$value['desc'],time()];
                                }
                            }
                            $model = LianWebModel::tableName();
                            $attr = ['type','title', 'url', 'image','desc','create_time'];
                            $res = Yii::$app->db->createCommand()->batchInsert($model, $attr, $type_data)->execute();
                        }
                    }
                }
            }
        }
    }

}